# -*- coding: utf-8 -*-
import sys
sys.modules[__name__].__dict__.clear()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.neighbors import KNeighborsClassifier
from sklearn.decomposition import PCA
from sklearn.metrics import confusion_matrix, accuracy_score
import matplotlib.gridspec as gridspec
from matplotlib.ticker import MaxNLocator
import matplotlib.colors as mcolors

# ������ʾѡ��Ա�������ʾ������
pd.set_option('display.max_columns', None)
# ��������
import matplotlib
matplotlib.rcParams['font.sans-serif']=['Microsoft YaHei']  # ������΢���ź�
matplotlib.rcParams['axes.unicode_minus']=False  # ���������ʾ����

# ����SCI������ɫ����
sci_colors = ['#0173B2', '#DE8F05', '#029E73', '#D55E00', '#CC78BC', '#CA9161', '#FBAFE4', '#949494', '#ECE133', '#56B4E9']

def load_data(file_path):
    """��������"""
    data = pd.read_csv(file_path, low_memory=False)
    print(f"ԭʼ������״: {data.shape}")
    return data

def remove_sparse_columns(data, threshold=0.3):
    """ɾ��ȱʧֵ�϶����"""
    null_sum = data.isnull().sum()
    keep = data.columns[null_sum < len(data) * threshold]  # ��������
    drop = data.columns[null_sum > len(data) * threshold]  # ��Ҫɾ������
    
    print(f"����������: {len(keep)}")
    print(f"ɾ��������: {len(drop)}")
    
    # ���ӻ�ȱʧֵ����ǰ��Ա�
    visualize_missing_values(data, threshold)
    
    data_clean = data.drop(columns=drop, inplace=False)
    print(f"\n�������������״: {data_clean.shape}")
    return data_clean

def visualize_missing_values(data, threshold):
    """���ӻ�ȱʧֵ����ǰ��Ա�"""
    # ����ÿ�е�ȱʧֵ����
    missing_ratio = data.isnull().mean().sort_values(ascending=False)
    
    # ��������ǰ��ĶԱ�ͼ
    plt.figure(figsize=(16, 8))
    
    # ����ǰ - �����е�ȱʧֵ����
    plt.subplot(1, 2, 1)
    # ֻ��ʾǰ30���У�����߿ɶ���
    top_missing = missing_ratio.head(30)
    bars = plt.bar(range(len(top_missing)), top_missing.values, color=sci_colors[0])
    plt.axhline(y=threshold, color=sci_colors[1], linestyle='-', linewidth=2, label=f'��ֵ ({threshold})')
    plt.title('����ǰ����ȱʧֵ���� (ǰ30��)', fontsize=14, fontweight='bold')
    plt.xlabel('��������', fontsize=12)
    plt.ylabel('ȱʧֵ����', fontsize=12)
    plt.xticks(range(len(top_missing)), top_missing.index, rotation=90, fontsize=8)
    plt.legend(fontsize=10)
    
    # ������ֵ��ǩ
    for i, bar in enumerate(bars):
        if top_missing.values[i] > 0.05:  # ֻΪ�ϴ��ֵ���ӱ�ǩ
            plt.text(i, top_missing.values[i] + 0.01, f'{top_missing.values[i]:.2f}', 
                    ha='center', va='bottom', fontsize=8, rotation=90)
    
    # ������ - �������е�ȱʧֵ����
    keep_cols = missing_ratio[missing_ratio < threshold].index
    missing_ratio_after = missing_ratio[missing_ratio < threshold]
    
    plt.subplot(1, 2, 2)
    # ֻ��ʾǰ30���У�����߿ɶ���
    top_missing_after = missing_ratio_after.head(30)
    bars = plt.bar(range(len(top_missing_after)), top_missing_after.values, color=sci_colors[2])
    plt.title('���������ȱʧֵ���� (ǰ30��)', fontsize=14, fontweight='bold')
    plt.xlabel('��������', fontsize=12)
    plt.ylabel('ȱʧֵ����', fontsize=12)
    plt.xticks(range(len(top_missing_after)), top_missing_after.index, rotation=90, fontsize=8)
    
    # ������ֵ��ǩ
    for i, bar in enumerate(bars):
        if top_missing_after.values[i] > 0.05:  # ֻΪ�ϴ��ֵ���ӱ�ǩ
            plt.text(i, top_missing_after.values[i] + 0.01, f'{top_missing_after.values[i]:.2f}', 
                    ha='center', va='bottom', fontsize=8, rotation=90)
    
    plt.tight_layout()
    plt.savefig('���ݴ���/����ɸ��.png', dpi=300, bbox_inches='tight')
    plt.show()

def check_data_types(data):
    """�����������"""
    object_features = data.dtypes[data.dtypes == 'object'].index
    other_features = data.dtypes[data.dtypes != 'object'].index
    
    print(f"�ַ������͵���������: {len(object_features)}")
    print(f"��ֵ���͵���������: {len(other_features)}")
    
    return object_features, other_features

def fix_chemical_components(data):
    """�޸���ѧ�ɷ����еĴ�������"""
    # ���洦��ǰ�����ݸ���
    data_before = data.copy()
    
    # �滻��ѧ�ɷ����е�*����
    for col in data.columns[1:20]:
        data.loc[:, col] = data[col].apply(lambda x: float(x.replace('*', '')) if isinstance(x, str) else x)
    
    # �ٴμ���ַ�����
    object_features_new = data.dtypes[data.dtypes == 'object'].index
    print(f"�������ַ������͵���������: {len(object_features_new)}")
    
    # ���ӻ�����ǰ��ĶԱ�
    visualize_chemical_components_fix(data_before, data)
    
    return data

def visualize_chemical_components_fix(data_before, data_after):
    """���ӻ���ѧ�ɷ�����ǰ��ĶԱ�"""
    # ѡ��ǰ5����ѧ�ɷ��н���չʾ
    chemical_cols = data_before.columns[1:6]
    
    # ����һ��������չʾ����ǰ��Ĳ���
    fig, ax = plt.subplots(figsize=(14, 8))
    ax.axis('tight')
    ax.axis('off')
    
    # ׼����������
    table_data = []
    headers = ['��������', '����ǰ��������', '����ǰ����ֵ������', '��������������', '���������ֵ������']
    
    for col in chemical_cols:
        before_type = str(data_before[col].dtype)
        after_type = str(data_after[col].dtype)
        
        # �������ֵ������
        before_non_numeric = sum(1 for x in data_before[col] if isinstance(x, str) and '*' in x)
        after_non_numeric = sum(1 for x in data_after[col] if isinstance(x, str) and '*' in x)
        
        table_data.append([col, before_type, before_non_numeric, after_type, after_non_numeric])
    
    # ��������
    table = ax.table(cellText=table_data, colLabels=headers, loc='center', cellLoc='center')
    table.auto_set_font_size(False)
    table.set_fontsize(12)
    table.scale(1, 1.8)
    
    # ���ñ�����ʽ
    for (i, j), cell in table.get_celld().items():
        if i == 0:  # ��ͷ��
            cell.set_text_props(weight='bold', color='white')
            cell.set_facecolor(sci_colors[0])
        elif j == 0:  # ��һ�У��������ƣ�
            cell.set_text_props(weight='bold')
            cell.set_facecolor('#f2f2f2')
        elif i % 2 == 1:  # ��������ɫ
            cell.set_facecolor('#f9f9f9')
    
    plt.title('��ѧ�ɷ���������ǰ��Ա�', fontsize=16, fontweight='bold', pad=20)
    plt.tight_layout()
    plt.savefig('���ݴ���/�˹��ɼ���������.png', dpi=300, bbox_inches='tight')
    plt.show()

def drop_manual_input_columns(data):
    """ɾ����ע���˹�������Ϣ�Լ�������Ϣ������ȫΪ0���У�"""
    columns_drop = ['���������', '��������', '�������ָ��','�����Ƿ񱨾�', '��ע', '����������', '�Ƿ���ǰ�رմ��', '����', 
                   '���߳���', '��ע����', '��¼����ʱ��', '��¼����������', '�������', '��������', 
                   '�����м����������', '�����м����������', '��ʼʱ��', '����ʱ��', '�ְ�����ʱ��', 
                   '�������׹��������', '�������׹����ѹ��', '�������׹��������', '�������׹����ѹ��',
                   'һ��ˮ����', '�����ܹ�ѹ��', '�����ȸ�ˮʱ��', '����40���������¶�', '����40���������¶�',
                   '�ְ��뿪ʱ��', '��ע��ʼʱ��', '��ע����ʱ��', '����5��������ʱ��', '����5��������ʱ��', '����10��������ʱ��', 
                   '����10��������ʱ��', '����20��������ʱ��', '����20��������ʱ��', '����30��������ʱ��', '����30��������ʱ��', 
                   '����40��������ʱ��', '����40��������ʱ��', '��������������ʱ��', '��������������ʱ��', '�����ֶ�����ʱ��1', 
                   '�����ֶ�����ʱ��2', '�����ֶ�����ʱ��3', '�����ֶ�����ʱ��4', '�����ֶ�����ʱ��5', '�����ֶ�����ʱ��6', 
                   '�����ֶ�����ʱ��7', '�����ֶ�����ʱ��1', '�����ֶ�����ʱ��2', '�����ֶ�����ʱ��3', '�����ֶ�����ʱ��4', 
                   '�����ֶ�����ʱ��5', '�����ֶ�����ʱ��6', '�����ֶ�����ʱ��7', '��¯����ʱ��', '�Ƿ���ǰ�رմ��',
                   '����̬����', '�Ƿ���β', '�Ƿ����', '��¼�޸�ʱ��', '��¼�޸�������']
    
    data_clean = data.drop(columns=columns_drop, inplace=False)
    print(f"ɾ���˹������к��������״: {data_clean.shape}")
    
    return data_clean

def clean_outliers(data, columns_to_process=None):
    """ʹ������ͼ�������������е���Ⱥ��"""
    if columns_to_process is None:
        # Ĭ�ϴ���������ֵ��
        numeric_columns = data.select_dtypes(include=[np.number]).columns.tolist()
    else:
        numeric_columns = columns_to_process
    
    print(f"������Ⱥ�������: {len(numeric_columns)}")
    
    # ���洦��ǰ�����ݸ���
    data_before = data.copy()
    
    # ��¼ÿ����Ⱥ������
    outliers_count_before = {}
    outliers_count_after = {}
    
    for selected_column in numeric_columns:
        # ����ͼɸѡ��Ⱥ��
        non_nan_data = data.dropna(subset=[selected_column])

        q1 = non_nan_data[selected_column].quantile(0.25)
        q3 = non_nan_data[selected_column].quantile(0.75)
        iqr = q3 - q1
        lower_bound = q1 - 1.5 * iqr
        upper_bound = q3 + 1.5 * iqr

        # ���㴦��ǰ����Ⱥ������
        outliers_count_before[selected_column] = ((data[selected_column] < lower_bound) | 
                                                 (data[selected_column] > upper_bound)).sum()

        # ����Ⱥ�㵥��ֵ�޸�ΪNaN
        data.loc[(data[selected_column] < lower_bound) | (data[selected_column] > upper_bound), selected_column] = np.nan
        
        # ���㴦�������Ⱥ��������Ӧ��Ϊ0��
        outliers_count_after[selected_column] = 0

    print(f"������Ⱥ����������״: {data.shape}")
    
    # ���ӻ���Ⱥ������ǰ��ĶԱ�
    visualize_outlier_cleaning(data_before, data, numeric_columns[:3], outliers_count_before, outliers_count_after)
    
    # �ر�չʾ"벺��¶ȡ�ƽ̨�¶ȡ���ʱ�����м���ȸ�ˮʱ��"
    special_columns = ['벺��¶�', 'ƽ̨�¶�', '��ʱ��', '�а��ȸ�ˮʱ��']
    visualize_special_columns(data_before, data, special_columns, outliers_count_before, outliers_count_after)
    
    return data

def visualize_outlier_cleaning(data_before, data_after, columns, outliers_before, outliers_after):
    """���ӻ���Ⱥ������ǰ��ĶԱ�"""
    n_cols = len(columns)
    fig, axes = plt.subplots(n_cols, 2, figsize=(16, 5 * n_cols))
    
    for i, col in enumerate(columns):
        # ����ǰ������ͼ
        sns.boxplot(x=data_before[col], ax=axes[i, 0], color=sci_colors[0])
        axes[i, 0].set_title(f'{col} - ����ǰ (��Ⱥ��: {outliers_before[col]}��)', fontsize=14, fontweight='bold')
        axes[i, 0].set_xlabel('��ֵ', fontsize=12)
        axes[i, 0].set_ylabel('Ƶ��', fontsize=12)
        
        # �����������ͼ
        sns.boxplot(x=data_after[col].dropna(), ax=axes[i, 1], color=sci_colors[2])
        axes[i, 1].set_title(f'{col} - ������ (��Ⱥ��: {outliers_after[col]}��)', fontsize=14, fontweight='bold')
        axes[i, 1].set_xlabel('��ֵ', fontsize=12)
        axes[i, 1].set_ylabel('Ƶ��', fontsize=12)
    
    plt.tight_layout()
    plt.savefig('���ݴ���/�쳣ֵ�޳�.png', dpi=300, bbox_inches='tight')
    plt.show()

def visualize_special_columns(data_before, data_after, columns, outliers_before, outliers_after):
    """���ӻ��ض��е���Ⱥ������ǰ��Ա�"""
    fig, axes = plt.subplots(3, 2, figsize=(16, 15))
    
    # չƽaxes�����Ա�������
    axes = axes.flatten()
    
    for i, col in enumerate(columns):
        # ����ǰ������ͼ
        sns.boxplot(x=data_before[col], ax=axes[i], color=sci_colors[i % len(sci_colors)])
        axes[i].set_title(f'{col} - ����ǰ (��Ⱥ��: {outliers_before[col]}��)', fontsize=14, fontweight='bold')
        axes[i].set_xlabel('��ֵ', fontsize=12)
        axes[i].set_ylabel('Ƶ��', fontsize=12)
        
        # �����������ͼ
        sns.boxplot(x=data_after[col].dropna(), ax=axes[i+len(columns)], color=sci_colors[(i+4) % len(sci_colors)])
        axes[i+len(columns)].set_title(f'{col} - ������ (��Ⱥ��: {outliers_after[col]}��)', fontsize=14, fontweight='bold')
        axes[i+len(columns)].set_xlabel('��ֵ', fontsize=12)
        axes[i+len(columns)].set_ylabel('Ƶ��', fontsize=12)
    
    plt.tight_layout()
    plt.savefig('���ݴ���/�ض����쳣ֵ�޳�.png', dpi=300, bbox_inches='tight')
    plt.show()

def merge_labels(data):
    """�ϲ��ǲ�����ʵ��ֵ��ǩ
    0 -> 0
    0.5 -> 1
    1 & 1.5 -> 2
    """
    # ���洦��ǰ�����ݸ���
    data_before = data.copy()
    
    columns_change = '�ǲ�����ʵ��ֵ'
    data[columns_change] = data[columns_change].replace({0.5: 1, 1: 2, 1.5: 2})
    
    # ���ӻ���ǩ�ϲ�ǰ��ĶԱ�
    visualize_label_merging(data_before, data, columns_change)
    
    return data

def visualize_label_merging(data_before, data_after, column):
    """���ӻ���ǩ�ϲ�ǰ��ĶԱ�"""
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 8))
    
    # ����ǰ�ı�ǩ�ֲ�
    before_counts = data_before[column].value_counts().sort_index()
    bars1 = ax1.bar(before_counts.index.astype(str), before_counts.values, color=sci_colors[:len(before_counts)])
    ax1.set_title('(a) ��ǩ�ϲ�ǰ', fontsize=16, fontweight='bold')
    ax1.set_xlabel('�ǲ����Ƶȼ�', fontsize=14)
    ax1.set_ylabel('������', fontsize=14)
    ax1.tick_params(axis='both', labelsize=12)
    
    # ������ֵ��ǩ
    for i, bar in enumerate(bars1):
        height = bar.get_height()
        ax1.text(bar.get_x() + bar.get_width()/2., height + 5,
                f'{height}', ha='center', va='bottom', fontsize=12, fontweight='bold')
    
    # ������ı�ǩ�ֲ�
    after_counts = data_after[column].value_counts().sort_index()
    bars2 = ax2.bar(after_counts.index.astype(str), after_counts.values, color=sci_colors[:len(after_counts)])
    ax2.set_title('(b) ��ǩ�ϲ���', fontsize=16, fontweight='bold')
    ax2.set_xlabel('�ǲ����Ƶȼ�', fontsize=14)
    ax2.set_ylabel('������', fontsize=14)
    ax2.tick_params(axis='both', labelsize=12)
    
    # ������ֵ��ǩ
    for i, bar in enumerate(bars2):
        height = bar.get_height()
        ax2.text(bar.get_x() + bar.get_width()/2., height + 5,
                f'{height}', ha='center', va='bottom', fontsize=12, fontweight='bold')
    
    # ���Ӻϲ�˵��
    ax2.text(0.5, -0.15, '��ǩ�ϲ�����: 0->0, 0.5->1, 1&1.5->2', 
             ha='center', va='center', transform=ax2.transAxes, 
             fontsize=12, bbox=dict(facecolor='#f9f9f9', alpha=0.5))
    
    plt.tight_layout()
    plt.savefig('���ݴ���/���Ƶȼ���ǩ�ϲ�.png', dpi=300, bbox_inches='tight')
    plt.show()

def save_data(data, output_file):
    """�������ݵ�CSV�ļ�"""
    data.to_csv(output_file, index=False, encoding='utf-8-sig')
    print(f"�����ѱ��浽: {output_file}")
    return output_file

def plot_boxplot(data, column_name):
    """��������ͼ"""
    plt.figure(figsize=(10, 6))
    data_num = data[[column_name]]
    sns.boxplot(data=data_num, fliersize=1, color=sci_colors[0])
    plt.title(f"{column_name}������ͼ", fontsize=14, fontweight='bold')
    plt.xlabel('��ֵ', fontsize=12)
    plt.ylabel('Ƶ��', fontsize=12)
    plt.tight_layout()
    plt.show()

def data_cleaning(input_file, output_file):
    """ִ��ȫ��������ϴ����"""
    # ����1: ��������
    data = load_data(input_file)
    
    # ����2: ɾ������ȱʧֵ����
    data = remove_sparse_columns(data)
    
    # ����3: �����������
    object_features, other_features = check_data_types(data)
    
    # ����4: �޸���ѧ�ɷ���
    data = fix_chemical_components(data)
    
    # ����5: ɾ���˹������к�������Ϣ
    data = drop_manual_input_columns(data)
    
    # ����6: ��ϴ��Ⱥ��
    columns_to_process = ['벺��¶�', 'ƽ̨�¶�', '��ʱ��', '�а��ȸ�ˮʱ��', '�ְ���������', '�ְ��뿪����', '�ְ���ˮ����', 
                      '��������ʱ��', '��������ʱ��', '�������ʱ�����м����ˮ����', '�������ʱ�����м����ˮ����', 
                      '����ս�ʱ�����м����ˮ����', '����ս�ʱ�����м����ˮ����', '����������ʱ�����¶�', 
                      '����5���������¶�', '����5���������¶�', '����10���������¶�', '����10���������¶�', 
                      '����20���������¶�', '����20���������¶�', '����30���������¶�', '����30���������¶�', 
                      '���������������¶�', '���������������¶�', 
                      '�����ֶ������¶�1', '�����ֶ������¶�2', '�����ֶ������¶�3', 
                      '�����ֶ������¶�4', '�����ֶ������¶�5', '�����ֶ������¶�6', '�����ֶ������¶�7', 
                      '�����ֶ������¶�1', '�����ֶ������¶�2', '�����ֶ������¶�3', 
                      '�����ֶ������¶�4', '�����ֶ������¶�5', '�����ֶ������¶�6', '�����ֶ������¶�7', 
                      '����ˮ�ܹ�ѹ��', 'ƽ��������', '�����м����βʱ����', '�����м����βʱ����', 
                      '�����а���ָ߶�', '�����а���ָ߶�', 'ת¯�ְ��¶�', 
                      'һ��ˮ������1', 'һ��ˮ������2', 'һ��ˮ������3', 'һ��ˮ������4', 'һ��ˮ������5', 
                      'һ��ˮ������6', 'һ��ˮ������7', 'һ��ˮ������8', 'һ��ˮ������9', 'һ��ˮ������10', 
                      'һ��ˮ����Ƶ��1', 'һ��ˮ����Ƶ��2', 'һ��ˮ����Ƶ��3', 'һ��ˮ����Ƶ��4', 'һ��ˮ����Ƶ��5', 
                      'һ��ˮ����Ƶ��6', 'һ��ˮ����Ƶ��7', 'һ��ˮ����Ƶ��8', 'һ��ˮ����Ƶ��9', 'һ��ˮ����Ƶ��10', 
                      '�����а�������1', '�����а�������2', '�����а�������1', '�����а�������2']
    data = clean_outliers(data, columns_to_process)
    
    # ����7: ��ǩ�ϲ�
    data = merge_labels(data)
    
    # ����8: ������ϴ�������
    save_data(data, output_file)
    
    print("������ϴ���!")
    return data

def get_defect_data(data):
    """��ȡȱ������"""
    # ����һ���������ж�ȱ���������0��1
    condition_def = data['ȱ�����'] == 1
    data_defect = data[condition_def]  # ʹ���������л���
    
    print(f'ȱ��������״: {data_defect.shape}')
    return data_defect

def split_known_unknown_defects(data_defect):
    """�ָ���֪��δ֪��ǩ��ȱ������"""
    # ����һ���������ж�ȱ�����
    condition_known = data_defect['�ǲ�����ʵ��ֵ'] != 42
    condition_unknown = data_defect['�ǲ�����ʵ��ֵ'] == 42
    data_defect_known = data_defect[condition_known]
    data_defect_unknown = data_defect[condition_unknown]
    
    print(f'��֪��ǩ��ȱ��������״: {data_defect_known.shape}')
    print(f'δ֪��ǩ��ȱ��������״: {data_defect_unknown.shape}')
    
    return data_defect_known, data_defect_unknown

def prepare_features_labels(data_known, data_unknown):
    """׼�������ͱ�ǩ����"""
    # �����������ǩ
    train_features = data_known.iloc[:, 1:111]
    train_labels = data_known.iloc[:, 111]
    test_features = data_unknown.iloc[:, 1:111]
    
    print(f'ѵ��������״: {train_features.shape}, ѵ����ǩ��״: {train_labels.shape}')
    print(f'����������״: {test_features.shape}')
    
    return train_features, train_labels, test_features

def preprocess_features(train_features, test_features):
    """Ԥ������������׼�������ȱʧֵ"""
    # ���洦��ǰ�����ݸ���
    train_features_before = train_features.copy()
    
    # ����ת��
    all_features = pd.concat((train_features, test_features))
    
    # �������Ͳ�Ϊobject�е�������ͨ���������������ŵ�0��ֵ�͵�λ��������׼������
    numeric_features = all_features.dtypes[all_features.dtypes != 'object'].index
    all_features[numeric_features] = all_features[numeric_features].apply(
                                            lambda x: (x - x.mean()) / (x.std()))
    # �ڱ�׼������֮�����о�ֵ��ʧ��������ǿ��Խ�ȱʧֵ����Ϊƽ��ֵ0
    all_features_before_filling = all_features.copy()
    all_features[numeric_features] = all_features[numeric_features].fillna(0)
    
    # "Dummy_na=True"��"na"��ȱʧֵ����Ϊ��Ч������ֵ����Ϊ�䴴��ָʾ������
    all_features = pd.get_dummies(all_features, dummy_na=True)
    
    print(f'Ԥ�������������״: {all_features.shape}')
    
    # ���ӻ���׼����ȱʧֵ���ǰ��ĶԱ�
    visualize_standardization(train_features_before, all_features[:len(train_features)], numeric_features[:3])
    visualize_missing_value_filling(all_features_before_filling, all_features, numeric_features[:3])
    
    return all_features

def visualize_standardization(data_before, data_after, columns):
    """���ӻ���׼��ǰ��ĶԱ�"""
    n_cols = len(columns)
    fig, axes = plt.subplots(n_cols, 2, figsize=(14, 4 * n_cols))
    
    for i, col in enumerate(columns):
        # ����ǰ�ķֲ�
        sns.histplot(data_before[col].dropna(), kde=True, ax=axes[i, 0])
        axes[i, 0].set_title(f'{col} - ��׼��ǰ')
        axes[i, 0].set_xlabel('ԭʼֵ')
        axes[i, 0].set_ylabel('Ƶ��')
        
        # ������ķֲ�
        sns.histplot(data_after[col].dropna(), kde=True, ax=axes[i, 1])
        axes[i, 1].set_title(f'{col} - ��׼����')
        axes[i, 1].set_xlabel('��׼��ֵ')
        axes[i, 1].set_ylabel('Ƶ��')
    
    plt.tight_layout()
    plt.savefig('���ݴ���/���ݱ�׼��.png', dpi=300, bbox_inches='tight')
    plt.show()

def visualize_missing_value_filling(data_before, data_after, columns):
    """���ӻ�ȱʧֵ���ǰ��ĶԱ�"""
    n_cols = len(columns)
    fig, axes = plt.subplots(n_cols, 2, figsize=(14, 4 * n_cols))
    
    for i, col in enumerate(columns):
        # ����ȱʧֵ����
        missing_before = data_before[col].isnull().mean() * 100
        missing_after = data_after[col].isnull().mean() * 100
        
        # ������ͼ - ����ǰ
        labels_before = [f'��Ч���� ({100-missing_before:.1f}%)', f'ȱʧֵ ({missing_before:.1f}%)']
        sizes_before = [100-missing_before, missing_before]
        axes[i, 0].pie(sizes_before, labels=labels_before, autopct='%1.1f%%', startangle=90)
        axes[i, 0].set_title(f'{col} - ���ǰ')
        
        # ������ͼ - ������
        labels_after = [f'��Ч���� (100.0%)', f'ȱʧֵ (0.0%)']
        sizes_after = [100, 0]
        axes[i, 1].pie(sizes_after, labels=labels_after, autopct='%1.1f%%', startangle=90)
        axes[i, 1].set_title(f'{col} - ����')
    
    plt.tight_layout()
    plt.savefig('���ݴ���/ȱʧֵ���.png', dpi=300, bbox_inches='tight')
    plt.show()

def apply_pca(features, variance_threshold=0.85):
    """Ӧ��PCA��ά"""
    # ���洦��ǰ�����ݸ���
    features_before = features.copy()
    
    # PCA��ά
    pca = PCA(n_components=variance_threshold, svd_solver='full')
    pca.fit(features)
    features_pca = pca.transform(features)
    
    print(f"ԭʼ������״: {features.shape}")
    print(f"��ά���������״: {features_pca.shape}")
    
    # ���ӻ�PCA��άǰ��ĶԱ�
    visualize_pca_reduction(features_before, features_pca, pca)
    
    return features_pca

def visualize_pca_reduction(features_before, features_after, pca):
    """���ӻ�PCA��άǰ��ĶԱ�"""
    fig = plt.figure(figsize=(16, 10))
    gs = gridspec.GridSpec(2, 2, height_ratios=[1, 1.5])
    
    # ����ǰ - ����ά��
    ax1 = plt.subplot(gs[0, 0])
    ax1.bar(range(features_before.shape[1]), [1] * features_before.shape[1], color=sci_colors[0])
    ax1.set_title('��άǰ����ά��', fontsize=14, fontweight='bold')
    ax1.set_xlabel('��������', fontsize=12)
    ax1.set_ylabel('������', fontsize=12)
    ax1.text(features_before.shape[1]/2, 0.5, f'��������: {features_before.shape[1]}', 
             ha='center', va='center', fontsize=12, bbox=dict(facecolor='white', alpha=0.8))
    
    # ������ - �ۻ����ͷ���
    ax2 = plt.subplot(gs[0, 1])
    cumulative_variance = np.cumsum(pca.explained_variance_ratio_)
    ax2.plot(range(1, len(cumulative_variance)+1), cumulative_variance, marker='o', color=sci_colors[1])
    ax2.axhline(y=0.85, color=sci_colors[3], linestyle='--', linewidth=2, label='��ֵ (85%)')
    ax2.set_title('PCA��ά���ۻ����ͷ���', fontsize=14, fontweight='bold')
    ax2.set_xlabel('���ɷ�����', fontsize=12)
    ax2.set_ylabel('�ۻ����ͷ����', fontsize=12)
    ax2.set_ylim(0, 1.05)
    ax2.legend(fontsize=10)
    
    # �ҵ��ﵽ85%��������ɷ�����
    n_components = np.argmax(cumulative_variance >= 0.85) + 1
    ax2.text(len(cumulative_variance)/2, 0.3, f'�ﵽ85%�����������ɷ���: {n_components}', 
             ha='center', va='center', fontsize=12, bbox=dict(facecolor='white', alpha=0.8))
    
    # ����ÿ�����ɷֵķ������
    ax3 = plt.subplot(gs[1, :])
    individual_variance = pca.explained_variance_ratio_
    bars = ax3.bar(range(1, len(individual_variance)+1), individual_variance, color=sci_colors[2])
    ax3.set_title('�����ɷֵķ������', fontsize=14, fontweight='bold')
    ax3.set_xlabel('���ɷ�', fontsize=12)
    ax3.set_ylabel('�������', fontsize=12)
    ax3.set_xticks(range(1, min(21, len(individual_variance)+1)))  # ֻ��ʾǰ20�����ɷ�
    
    # ������ֵ��ǩ
    for i, bar in enumerate(bars[:20]):  # ֻΪǰ20�����ɷ����ӱ�ǩ
        height = bar.get_height()
        if height > 0.01:  # ֻΪ�ϴ��ֵ���ӱ�ǩ
            ax3.text(i+1, height + 0.005, f'{height:.3f}', ha='center', va='bottom', fontsize=8, rotation=90)
    
    plt.tight_layout()
    plt.savefig('���ݴ���/���ݽ�ά.png', dpi=300, bbox_inches='tight')
    plt.show()

def train_knn_model(X_train, y_train):
    """ѵ��KNNģ��"""
    # ��ʼ��KNN������
    knn_classifier = KNeighborsClassifier(n_neighbors=3)
    # ѵ��ģ��
    knn_classifier.fit(X_train, y_train)
    
    # ����ѵ�����ϵ�׼ȷ��
    train_predictions = knn_classifier.predict(X_train)
    train_accuracy = accuracy_score(y_train, train_predictions)
    
    print(f"KNNģ��ѵ�����! ѵ����׼ȷ��: {train_accuracy:.4f}")
    
    return knn_classifier, train_accuracy

def predict_and_save(model, features_test, test_features_index, train_features, train_labels, test_features, train_accuracy):
    """��ģ��Ԥ�Ⲣ������"""
    # ����Ԥ��
    predictions = model.predict(features_test)
    
    # ��Ԥ����תΪDataFrame����������ͬ��index
    predictions_df = pd.DataFrame(predictions, columns=['�ǲ�����ʵ��ֵ'], index=test_features_index.index)
    
    # ��Ԥ�����������ϲ�
    data_unknown_new = pd.concat([test_features, predictions_df], axis=1)
    data_unknown_new['ȱ�����'] = 1
    data_unknown_new['������'] = 1
    
    # �ϲ���֪��ǩ������
    data_known_new = pd.concat([train_features, train_labels], axis=1)
    data_known_new['ȱ�����'] = 1
    data_known_new['������'] = 1
    
    # ��Ԥ�������ӵ�ԭʼDataFrame
    data_defect_exist = pd.concat([data_known_new, data_unknown_new], axis=0)
    
    # ���ӻ�α��ǩ����ǰ��ĶԱ�
    visualize_pseudo_labeling(train_labels, data_defect_exist['�ǲ�����ʵ��ֵ'], train_accuracy)
    
    print(f'Ԥ����ɣ�����ȱ��������״: {data_defect_exist.shape}')
    
    return data_defect_exist

def visualize_pseudo_labeling(labels_before, labels_after, train_accuracy):
    """���ӻ�α��ǩ����ǰ��ĶԱ�"""
    fig = plt.figure(figsize=(16, 10))
    gs = gridspec.GridSpec(2, 2, height_ratios=[1.5, 1])
    
    # ����ǰ�ı�ǩ�ֲ�
    ax1 = plt.subplot(gs[0, 0])
    before_counts = labels_before.value_counts().sort_index()
    bars1 = ax1.bar(before_counts.index.astype(str), before_counts.values, color=sci_colors[:len(before_counts)])
    ax1.set_title('(a) ����α��ǩǰ', fontsize=16, fontweight='bold')
    ax1.set_xlabel('�ǲ����Ƶȼ�', fontsize=14)
    ax1.set_ylabel('������', fontsize=14)
    ax1.tick_params(axis='both', labelsize=12)
    
    # ������ֵ��ǩ
    for i, bar in enumerate(bars1):
        height = bar.get_height()
        ax1.text(bar.get_x() + bar.get_width()/2., height + 5,
                f'{height}', ha='center', va='bottom', fontsize=12, fontweight='bold')
    
    # ������ı�ǩ�ֲ�
    ax2 = plt.subplot(gs[0, 1])
    after_counts = labels_after.value_counts().sort_index()
    bars2 = ax2.bar(after_counts.index.astype(str), after_counts.values, color=sci_colors[:len(after_counts)])
    ax2.set_title('(b) ����α��ǩ��', fontsize=16, fontweight='bold')
    ax2.set_xlabel('�ǲ����Ƶȼ�', fontsize=14)
    ax2.set_ylabel('������', fontsize=14)
    ax2.tick_params(axis='both', labelsize=12)
    
    # ������ֵ��ǩ
    for i, bar in enumerate(bars2):
        height = bar.get_height()
        ax2.text(bar.get_x() + bar.get_width()/2., height + 5,
                f'{height}', ha='center', va='bottom', fontsize=12, fontweight='bold')
    
    # ����KNNģ������ָ��
    ax3 = plt.subplot(gs[1, :])
    ax3.axis('off')
    
    # ������������
    performance_data = [
        ['KNNģ�Ͳ���', 'n_neighbors=3'],
        ['ѵ����׼ȷ��', f'{train_accuracy:.4f}'],
        ['α��ǩ����', f'{len(labels_after) - len(labels_before)}'],
        ['��ǩ�ֲ��仯', f'��{len(before_counts)}�����{len(after_counts)}�����']
    ]
    
    # ��������
    table = ax3.table(
        cellText=performance_data,
        colLabels=['ָ��', 'ֵ'],
        loc='center',
        cellLoc='center'
    )
    
    # ���ñ�����ʽ
    table.auto_set_font_size(False)
    table.set_fontsize(12)
    table.scale(1, 1.5)
    
    for (i, j), cell in table.get_celld().items():
        if i == 0:  # ��ͷ��
            cell.set_text_props(weight='bold', color='white')
            cell.set_facecolor(sci_colors[0])
        elif j == 0:  # ��һ�У�ָ�����ƣ�
            cell.set_text_props(weight='bold')
            cell.set_facecolor('#f2f2f2')
    
    plt.tight_layout()
    plt.savefig('���ݴ���/���Ƶȼ�α��ǩ����.png', dpi=300, bbox_inches='tight')
    plt.show()

def knn_labeling(cleaned_data, output_file):
    """ִ��KNN��ǩԤ���ȫ������"""
    # ����1: ��ȡȱ������
    data_defect = get_defect_data(cleaned_data)
    
    # ����2: �ָ���֪��δ֪��ǩ��ȱ������
    data_defect_known, data_defect_unknown = split_known_unknown_defects(data_defect)
    
    # ����3: ׼�������ͱ�ǩ
    train_features, train_labels, test_features = prepare_features_labels(data_defect_known, data_defect_unknown)
    
    # ����4: Ԥ��������
    all_features = preprocess_features(train_features, test_features)
    
    # ����5: Ӧ��PCA��ά
    features_pca = apply_pca(all_features)
    
    # ����6: ׼��ѵ���Ͳ�������
    n_train = train_features.shape[0]
    features_train = features_pca[:n_train, :]
    labels_train = train_labels
    features_test = features_pca[n_train:, :]
    features_test_index = all_features.iloc[n_train:, :]
    
    # ����7: ѵ��KNNģ��
    knn_model, train_accuracy = train_knn_model(features_train, labels_train)
    
    # ����8: Ԥ��δ֪��ǩ
    data_defect_exist = predict_and_save(knn_model, features_test, features_test_index, train_features, train_labels, test_features, train_accuracy)
    
    # ����9: ��ȡ��ȱ������
    # ��ȡ��ȱ������
    condition_no_def = cleaned_data['ȱ�����'] == 0
    data_no_defect = cleaned_data[condition_no_def]
    print(f'��ȱ��������״: {data_no_defect.shape}')
    
    # ����10: �ϲ���������
    # �ϲ���ȱ�ݺ���ȱ������
    data_all = pd.concat([data_defect_exist, data_no_defect], axis=0)
    print(f'�ϲ������������״: {data_all.shape}')
    
    # ����11: �������Ԥ���ǩ������
    save_data(data_all, output_file)
    
    print("KNN��ǩԤ�����!")
    return data_all

def create_summary_table():
    """�������ݴ�������ժҪ��"""
    # ������������
    steps = [
        '����ɸ��', 
        '�˹��ɼ���������', 
        '�쳣ֵ�޳�', 
        '���ݱ�׼��', 
        'ȱʧֵ���', 
        '���ݽ�ά', 
        '���Ƶȼ���ǩ�ϲ�', 
        '���Ƶȼ�α��ǩ����'
    ]
    
    visualization_types = [
        '����ͼ', 
        '����', 
        '����ͼ', 
        'ֱ��ͼ', 
        '��ͼ', 
        '����ͼ+����ͼ', 
        '����ͼ', 
        '����ͼ+����'
    ]
    
    reasons = [
        'ֱ��չʾ����ȱʧֵ�����仯��ͻ����ֵ��',
        '����չʾ�������ͺʹ������������仯��ʹ�ñ�����ǿ�ɶ���',
        'ֱ��չʾ���ݷֲ�����Ⱥ�����Ч����������Ⱥ��������ע',
        'չʾ���ݷֲ���׼��ǰ��ı仯��ʹ��ֱ��ͼ+�ܶ�������ǿ���ӻ�Ч��',
        'ֱ��չʾȱʧֵ��������ʹ�ñ�ͼͻ�����ǰ��Ա�',
        'չʾ��ά�����з�������ʵ��ۻ��仯�������Ӹ����ɷֵķ������',
        'չʾ��ǩ�ϲ�ǰ������ֲ��仯��������ֵ��ǩ��ǿ�ɶ���',
        'չʾα��ǩ����ǰ��������ֲ��仯������ģ������ָ�����'
    ]
    
    # ��������
    fig, ax = plt.subplots(figsize=(14, 10))
    ax.axis('tight')
    ax.axis('off')
    
    table = ax.table(
        cellText=[[steps[i], visualization_types[i], reasons[i]] for i in range(len(steps))],
        colLabels=['���ݴ�������', '���ӻ�����', 'ѡ��ԭ��'],
        loc='center',
        cellLoc='center'
    )
    
    table.auto_set_font_size(False)
    table.set_fontsize(10)
    table.scale(1, 1.8)
    
    # ���ñ�����ʽ
    for (i, j), cell in table.get_celld().items():
        if i == 0:  # ��ͷ��
            cell.set_text_props(weight='bold', color='white')
            cell.set_facecolor(sci_colors[0])
        elif j == 0:  # ��һ�У��������ƣ�
            cell.set_text_props(weight='bold')
            cell.set_facecolor('#f2f2f2')
        elif i % 2 == 1:  # ��������ɫ
            cell.set_facecolor('#f9f9f9')
    
    plt.title('���ݴ�����������ӻ�����ѡ��ժҪ', fontsize=16, fontweight='bold', pad=20)
    plt.tight_layout()
    plt.savefig('���ݴ���/���ӻ�����ѡ��ժҪ.png', dpi=300, bbox_inches='tight')
    plt.show()

def visualize_four_special_columns(data):
    """��"벺��¶ȡ�ƽ̨�¶ȡ���ʱ�����м���ȸ�ˮʱ��"����һ��ͼ��"""
    special_columns = ['벺��¶�', 'ƽ̨�¶�', '��ʱ��', '�а��ȸ�ˮʱ��']
    
    # �����Щ���Ƿ���������
    available_columns = [col for col in special_columns if col in data.columns]
    
    if len(available_columns) == 0:
        print("ָ�����в���������")
        return
    
    fig, axes = plt.subplots(2, 2, figsize=(16, 12))
    axes = axes.flatten()
    
    for i, col in enumerate(available_columns):
        # ��������ͼ
        sns.boxplot(x=data[col].dropna(), ax=axes[i], color=sci_colors[i])
        axes[i].set_title(f'{col}�ֲ�', fontsize=14, fontweight='bold')
        axes[i].set_xlabel('��ֵ', fontsize=12)
        axes[i].set_ylabel('Ƶ��', fontsize=12)
        
        # ����ͳ����Ϣ
        stats = data[col].describe()
        info_text = f"��ֵ: {stats['mean']:.2f}\n��λ��: {stats['50%']:.2f}\n��׼��: {stats['std']:.2f}\n��Сֵ: {stats['min']:.2f}\n���ֵ: {stats['max']:.2f}"
        axes[i].text(0.05, 0.95, info_text, transform=axes[i].transAxes, 
                    verticalalignment='top', bbox=dict(boxstyle='round', facecolor='white', alpha=0.8))
    
    plt.tight_layout()
    plt.savefig('���ݴ���/�ĸ������зֲ�.png', dpi=300, bbox_inches='tight')
    plt.show()

def main():
    # �����ļ�·��
    input_file = 'Data_V1.csv'
    cleaned_file = 'Data_V2.csv'
    final_file = 'Data_V3.csv'
    
    # ��һ���֣�������ϴ
    print("="*50)
    print("��ʼ������ϴ...")
    print("="*50)
    cleaned_data = data_cleaning(input_file, cleaned_file)
    
    # ���ӻ��ĸ�������
    visualize_four_special_columns(cleaned_data)
    
    # �ڶ����֣�KNN��ǩԤ��
    print("\n"+"="*50)
    print("��ʼKNN��ǩԤ��...")
    print("="*50)
    final_data = knn_labeling(cleaned_data, final_file)
    
    # �������ӻ�����ѡ��ժҪ��
    create_summary_table()
    
    print("\n"+"="*50)
    print("ȫ���������!")
    print(f"���������ѱ����� {final_file}")
    print("="*50)

if __name__ == "__main__":
    main()